from downloader import Downloader, RelationshipDownloader
import content_parser
from timer import Timer
from utilities import mode_add_one
from configs import ACC_ACCESS_LIMIT, ACC_RESET_TIME
import configs
import time
from daos import Account, Keyword, Timeline, Seed
import daos as DAOS
from logger import Logger


class Crawler(object):
    def __init__(self):
        self.keywords = Keyword.get_keywords()
        self.timelines = []

        self.downloader = Downloader()
        self.parser = content_parser.Parser()
        self.life_timer = Timer()

        self.accounts = Account.get_accounts()
        self.account = None
        self.acc_index = 0
        self.acc_timer = None

        self.log = Logger('latest_crawling_time.log', 'Crawler').logger

    def init_account(self):
        if self.account is not None:
            return 1
        self.account = self.accounts[self.acc_index]
        self.acc_index = mode_add_one(self.acc_index, len(self.accounts))
        self.acc_timer = Timer()
        print 'Loading account:%s ...' % (self.account.username,)

    def change_account(self):
        index = self.acc_index
        while True:
            self.account = self.accounts[self.acc_index]
            self.acc_index = mode_add_one(self.acc_index, len(self.accounts))
            if self.account.count > ACC_ACCESS_LIMIT:
                if self.acc_index == index:
                    self.reset_accounts()

                continue
            else:
                print 'Change to account-%s ...' % (self.account.username,)
                return 0

    def reset_accounts(self):
        while True:
            elapsed_time = self.acc_timer.click()
            if elapsed_time > ACC_RESET_TIME:
                for account in self.accounts:
                    account.count = 0
                print 'Reset all the accounts...'
                self.acc_timer.reset()
                return 0
            else:
                sleep = ACC_RESET_TIME - elapsed_time + 5
                print 'Sleep %d seconds...' % (sleep,)
                time.sleep(sleep)

    def dump_timelines(self):
        print 'Dumping timelines to database...'
        Timeline.dump(self.timelines)
        self.timelines = []  # release memory

    def parse_timelines(self, timeline_panel):
        return self.parser.parse_timelines(timeline_panel)

    def download_one_page_timelines(self, keyword, page):
        while True:
            timeline_panel = self.downloader.download(self.account, keyword, page)

            if timeline_panel is None:
                print 'Timeline pannel is empty!'  # for debugx
                self.change_account()
                continue

            return timeline_panel

    def download_related_timelines(self, keyword):
        all_cards = []
        page = 1

        while True:
            print 'Getting page %d...' % (page,)
            timeline_panel = self.download_one_page_timelines(keyword, page)

            try:
                cards = timeline_panel['cards']
            except KeyError:
                self.log.warning('Dirty data...')
                self.log.warning(keyword)
                self.log.warning(timeline_panel)
                if self.handle_KeyError_for_timeline_panel(timeline_panel) is True:
                    return all_cards
                else:
                    continue
            if len(cards) == 0:
                return all_cards
            else:
                all_cards.extend(cards)
                page += 1

    def handle_KeyError_for_timeline_panel(self, timeline_panel):
        errno = timeline_panel['errno']

        if errno == u'1001030042':  # According to the Policy, the search results are prohibited
            return True
        elif errno == -200:
            self.change_account()
            return False
        else:
            return False

    def search_and_get_related_timelines(self):
        self.init_account()

        for keyword in self.keywords:
            keyword = keyword.keyword
            print 'Get timelines about "%s"' % (keyword,)
            all_cards = self.download_related_timelines(keyword)
            timelines, earliest_post_time = self.parse_timelines(all_cards)
            self.timelines.extend(timelines)
            self.log.info('Keyword: %s -- Earliest Post Time: %s' % (keyword, earliest_post_time))
            self.dump_timelines()

    def update_keywords(self):
        print 'Updating keywords...'
        Keyword.disable_keywords(self.keywords)
        self.keywords = Keyword.get_keywords()


class RelationshipCrawler(object):
    def __init__(self):
        self.seed_list = []
        self.followee_list = []
        self.follower_list = []

        self.downloader = RelationshipDownloader()
        self.parser = content_parser.RelationshipParser()

        self.accounts = []
        self.account = None
        self.acc_index = 0
        self.acc_timer = None

        self.log = Logger('relationship_crawler.log', 'RelationshipCrawler').logger
        self.init_account()
        self.seed_list = Seed.get_seeds()

    def update_seeds(self):
        self.disable_seeds()
        self.seed_list = Seed.get_seeds()

    def disable_seeds(self):
        Seed.disable_seeds(self.seed_list)

    def init_account(self):
        self.accounts = Account.get_accounts()
        self.account = self.accounts[self.acc_index]
        self.acc_index = mode_add_one(self.acc_index, len(self.accounts))
        self.acc_timer = Timer()
        print 'Loading account:%s ...' % (self.account.username,)

    def change_account(self):
        index = self.acc_index
        while True:
            self.account = self.accounts[self.acc_index]
            self.acc_index = mode_add_one(self.acc_index, len(self.accounts))
            if self.account.count > ACC_ACCESS_LIMIT:
                if self.acc_index == index:
                    self.reset_accounts()

                continue
            else:
                print 'Change to account -- %s ...' % (self.account.username,)
                return 0

    def reset_accounts(self):
        while True:
            elapsed_time = self.acc_timer.click()
            print 'Accouts have run for %d seconds!' % (elapsed_time,)
            if elapsed_time > ACC_RESET_TIME:
                for account in self.accounts:
                    account.count = 0
                print 'Reset all the accounts...'
                self.acc_timer.reset()
                return 0
            else:
                sleep = ACC_RESET_TIME - elapsed_time + 5
                print 'Sleep %d seconds...' % (sleep,)
                time.sleep(sleep)

    def get_followees_for_one_page(self, seed, page):
        while True:
            followees = self.downloader.download_relationship(self.account, seed, page, 'followers')
            # 'followers' corresponds to get followees

            if followees is None:
                self.log.warning('Get no followeEs!!!')
                self.change_account()
                continue
            else:
                return followees

    def get_followers_for_one_page(self, seed, page):
        while True:
            followers = self.downloader.download_relationship(self.account, seed, page, 'fans')
            # 'fans' corresponds to get followers

            if followers is None:
                self.log.warning('Get no followeRs!!!')
                self.change_account()
                continue
            else:
                return followers

    def get_followees_for_one_user(self, seed):
        followee_list = []

        page = 1
        while True:
            print 'Get followeE page %d for user %s...' % (page, seed.uid)
            followees = self.get_followees_for_one_page(seed, page)

            try:
                cards = followees['cards']
                cards[0]['card_group']
            except KeyError:
                yield followee_list
                print 'No followeEs any more!'
                raise StopIteration
            for card in cards:
                if 'card_group' not in card:
                    continue
                followee_list.extend(card['card_group'])
            yield followee_list
            if not page % 100:
                followee_list = []
            page += 1

    def get_followers_for_one_user(self, seed):
        follower_list = []

        page = 1
        while True:
            print 'Get followeR page %d for user %s...' % (page, seed.uid)
            followers = self.get_followers_for_one_page(seed, page)

            try:
                cards = followers['cards']
                cards[0]['card_group']
            except KeyError:
                yield follower_list
                print 'No followeRs any more!'
                raise StopIteration
            for card in cards:
                if 'card_group' not in card:
                    continue
                follower_list.extend(card['card_group'])  # sometimes there is no card_group

            yield follower_list
            if not page % 100:
                follower_list = []
            page += 1

    def dump_followees(self, force=False):
        if force is False and len(self.followee_list) < configs.RELATIONSHIP_BUFFER:
            return 0

        print 'Dump followeEs to database...'
        dump_num = DAOS.Followee.dump(self.followee_list)
        self.log.info('Dump %d followeEs into database!' % (dump_num,))
        self.followee_list = []

    def dump_followers(self, force=False):
        if force is False and len(self.follower_list) < configs.RELATIONSHIP_BUFFER:
            return 0

        print 'Dump followeRs to database...'
        dump_num = DAOS.Follower.dump(self.follower_list)
        self.log.info('Dump %d followeRs into database!' % (dump_num,))
        self.follower_list = []

    def get_relationships(self):
        if not len(self.seed_list):
            raise Exception('No seed!')
        for seed in self.seed_list:
            uid_set = set()

            print 'Get followeEs for user %s...' % (seed.uid,)
            for followees in self.get_followees_for_one_user(seed):
                print 'Parse followeEs for user %s...' % (seed.uid,)
                followees = self.parser.parse_followees(followees, seed.uid)
                if len(followees) > 0 and followees[0].uid in uid_set:
                    break
                else:
                    uid_set.update([f.uid for f in followees])
                    self.followee_list.extend(followees)

                self.dump_followees()

            self.dump_followees(True)
            uid_set.clear()

            print 'Get followeRs for user %s...' % (seed.uid,)
            for followers in self.get_followers_for_one_user(seed):
                print 'Parse followeRs for user %s...' % (seed.uid,)
                followers = self.parser.parse_followers(followers, seed.uid)
                if len(followers) > 0 and followers[0].uid in uid_set:
                    break
                else:
                    uid_set.update([f.uid for f in followers])
                    self.follower_list.extend(followers)

                self.dump_followers()

            self.dump_followers(True)







